import re
import random
import sys
import time
import datetime
import json
import math
import os
import pprint
import platform
import requests

from bs4 import BeautifulSoup
from selenium import webdriver
from pymongo import MongoClient
from multiprocessing import Process
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

net = sys.argv[1] if len(sys.argv) == 2 else 'localhost'

sys.setrecursionlimit(1000000)

client = MongoClient(net, 27017, connect = False)

toutiao_user_db = client['toutiao_user_db']
toutiao_user_coll = toutiao_user_db['toutiao_user_coll']

if platform.system() == 'Linux':
    chrome_driver_path = os.getcwd()[:os.getcwd().rfind('/') + 1] + 'chromedriver_linux'
else:
    chrome_driver_path = os.getcwd()[:os.getcwd().rfind('/') + 1] + 'chromedriver'


def new_driver():
    options = webdriver.ChromeOptions()
    options.add_argument('--headless')
    options.add_argument('--disable-gpu')
    options.add_argument('--disable-images')

    if platform.system() == 'Linux':
        options.binary_location = '/snap/bin/chromium'

    desired_capabilities = options.to_capabilities()
    desired_capabilities['loggingPrefs'] = {'performance': 'ALL'}
    driver = webdriver.Chrome(chrome_driver_path, desired_capabilities = desired_capabilities)

    return driver


def parse_signature(id, driver):
    url = 'https://www.toutiao.com/c/user/relation/' + str(id) + '/'
    driver.get(url)

    for entry in driver.get_log('performance')[::-1]:
        message = json.loads(entry['message'])['message']['params']

        if not 'request' in message:
            continue

        if not 'url' in message['request']:
            continue

        url = message['request']['url']

        if url.find('https://www.toutiao.com/c/user/following/?user_id') != -1:
            if re.findall('_signature=(.+)', url):
                return re.findall('_signature=(.+)', url)[0]


def parse_following(user_id = '', cursor = 0, signature = '', driver = None):
    if not user_id:
        return

    if not driver:
        driver = new_driver()

    if not signature:
        signature = parse_signature(user_id, driver)
        print(signature)
        time.sleep(3)

    url = 'https://www.toutiao.com/c/user/following/?user_id=' + str(user_id) + '&cursor=' + str(cursor) + '&count=500&_signature=' + signature

    try:
        driver.get(url)
    except:
        print('访问 www.toutiao.com 的请求遭到拒绝')
        return

    if len(driver.page_source.strip()) < 100:
        print('页面为空')
        return

    soup = BeautifulSoup(driver.page_source, 'lxml')

    try:
        resp = json.loads(soup.body.get_text())
    except:
        return

    data = resp['data']

    for item in data:
        user = {
            'id': item['user_id'],
            'name': item['name']
        }

        if not toutiao_user_coll.find_one({'id': user['id']}):
            toutiao_user_coll.insert_one(user)
            pprint.pprint(user['name'])

    time.sleep(2)

    for i in toutiao_user_coll.aggregate([{'$sample': {'size': 1}}]):
        parse_following(user_id = i['id'], driver = driver)


# def users(id, driver = None):
#     if not driver:
#         driver = new_driver()
#
#     try:
#         driver.get('https://www.toutiao.com/c/user/relation/' + id + '/?tab=following')
#         time.sleep(1)
#     except:
#         return users(id, driver)
#
#     soup = BeautifulSoup(driver.page_source, 'lxml')
#
#     for item in soup.select('.media li'):
#         id = re.findall('\d+', item.find('a').get('href'))[0]
#
#         item = {
#             'id': id,
#             'name': item.find('h3').get_text()
#         }
#
#         if not toutiao_user_coll.find_one({'id': item['id']}):
#             toutiao_user_coll.insert_one(item)
#             pprint.pprint(item)
#
#     for i in toutiao_user_coll.aggregate([{'$sample': {'size': 1}}]):
#         users(i['id'], driver)


if __name__ == '__main__':
    pool = []

    for i in toutiao_user_coll.aggregate([{'$sample': {'size': 30}}]):
        p = Process(target = parse_following, args = (i['id'],))
        pool.append(p)

    for p in pool:
        p.start()

    for p in pool:
        p.join()
